{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "Linear probing with ImageGPT.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "mount_file_id": "1w9fhdpCs3M7rTgxfiYj8217XDVPunCTu",
      "authorship_tag": "ABX9TyOvf3Cmz5xWWBJP3MFwpNRJ",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "8e27071d0c764607b5ca313dd8cf91ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_edb3be86b57d44339d7ded0c46907d79",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_16231fa674ce46f58e357a4c041c732a",
              "IPY_MODEL_79fc7f3ea9764d11b666e34c3d0b9841",
              "IPY_MODEL_433d6446de20455aaa6cd723dcf15d7f"
            ]
          }
        },
        "edb3be86b57d44339d7ded0c46907d79": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "16231fa674ce46f58e357a4c041c732a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_28de0ea9f2f1492697149582659b46ed",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: ",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9a14333682a24325a8e44ec1a75b4912"
          }
        },
        "79fc7f3ea9764d11b666e34c3d0b9841": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b3c6b3d6c6c34ea4bf33a1fd044d15dc",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1546,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1546,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e8c79f02de81453cbad571b3f025795a"
          }
        },
        "433d6446de20455aaa6cd723dcf15d7f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_98e3d575d682457b83284d9e06344a54",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 3.77k/? [00:00&lt;00:00, 83.3kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_30a4c61d98f3415bbe2f4a828aa9c446"
          }
        },
        "28de0ea9f2f1492697149582659b46ed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9a14333682a24325a8e44ec1a75b4912": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b3c6b3d6c6c34ea4bf33a1fd044d15dc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e8c79f02de81453cbad571b3f025795a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "98e3d575d682457b83284d9e06344a54": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "30a4c61d98f3415bbe2f4a828aa9c446": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c1a3e8abfd2044bcac7dda53ecb7e078": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_63356a99831e4c26a691c91c56c10027",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_0526b477d4c74cc18e90448f14ecb41f",
              "IPY_MODEL_6169f7d9f0804680bdd331389cd6fb1d",
              "IPY_MODEL_7f73a39a438f4381af9ae318edf6535a"
            ]
          }
        },
        "63356a99831e4c26a691c91c56c10027": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0526b477d4c74cc18e90448f14ecb41f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9b1b445b45bc40b98a3255dbdeb21c08",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: ",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ab5b5c1d7fb647948c0e76a4c9e323df"
          }
        },
        "6169f7d9f0804680bdd331389cd6fb1d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_3244def7ec0444d7a50ed4962448c8f4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 773,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 773,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1312814b7aa34c16990d910411287537"
          }
        },
        "7f73a39a438f4381af9ae318edf6535a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_944bb179e4124334a525621bbde83941",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.50k/? [00:00&lt;00:00, 41.9kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ee18abc5b6f545b3800b2426fabc9057"
          }
        },
        "9b1b445b45bc40b98a3255dbdeb21c08": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ab5b5c1d7fb647948c0e76a4c9e323df": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3244def7ec0444d7a50ed4962448c8f4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1312814b7aa34c16990d910411287537": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "944bb179e4124334a525621bbde83941": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ee18abc5b6f545b3800b2426fabc9057": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c51333916bfb45e3b1542cacdee81661": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_7d31b37ddbfb473580f0bc15d087a214",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_6de8bd510cfe49aea9e5d8adfccedecf",
              "IPY_MODEL_1955773b8bfc4d1fa7bf5acb10701991",
              "IPY_MODEL_1b2bac60febc49319905da1fea9143ba"
            ]
          }
        },
        "7d31b37ddbfb473580f0bc15d087a214": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6de8bd510cfe49aea9e5d8adfccedecf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c52c0eeb9f70499ebefcf87c7e11087f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9f5237745f434e72a79842062e55eade"
          }
        },
        "1955773b8bfc4d1fa7bf5acb10701991": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_033efd3e870f435c88b65f1760e716e1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 170498071,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 170498071,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cfa7811d658b46cbafb47edefcc00e98"
          }
        },
        "1b2bac60febc49319905da1fea9143ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_730d3f6e486747a7941f25be8eeeb0e0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 170M/170M [00:07&lt;00:00, 29.2MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b2a6c9eecb774c938ebacf80c38cab4b"
          }
        },
        "c52c0eeb9f70499ebefcf87c7e11087f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9f5237745f434e72a79842062e55eade": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "033efd3e870f435c88b65f1760e716e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cfa7811d658b46cbafb47edefcc00e98": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "730d3f6e486747a7941f25be8eeeb0e0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b2a6c9eecb774c938ebacf80c38cab4b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8e9a09c303784195bd55a227adfa9985": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_49de16113879490ba222b571a503ad72",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2929c29806cc449f81790c61d616724c",
              "IPY_MODEL_cee5a680445b4b599a7e2d5605691917",
              "IPY_MODEL_15e3fec9d0ee43999a7a1ec21fd5d5e6"
            ]
          }
        },
        "49de16113879490ba222b571a503ad72": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2929c29806cc449f81790c61d616724c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9e73febb198c4a32a1769eb7fda380f3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a429ada21ede47c9b750b618c66bfa65"
          }
        },
        "cee5a680445b4b599a7e2d5605691917": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a58281f8e21e47f0be708d2555c1bd62",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_20a73cda113241d79535974214c8e2ec"
          }
        },
        "15e3fec9d0ee43999a7a1ec21fd5d5e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ac7cf13830094d37813ed65c0cf0ea20",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 49965/0 [00:45&lt;00:00, 826.40 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f18f3fc6c4124d54935075f5e45f1f27"
          }
        },
        "9e73febb198c4a32a1769eb7fda380f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a429ada21ede47c9b750b618c66bfa65": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a58281f8e21e47f0be708d2555c1bd62": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "20a73cda113241d79535974214c8e2ec": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": "20px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ac7cf13830094d37813ed65c0cf0ea20": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f18f3fc6c4124d54935075f5e45f1f27": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f3b96aa457bd498d89c6a4c18bd8695c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_539a3ea318c4474289bc8d786086e4bf",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7ff18e2d227b416ca665fa1f94116b1b",
              "IPY_MODEL_5120ce6bb16b44918287396bf92d14c1",
              "IPY_MODEL_01645d12521d436d87b92fb6c5e5b3ea"
            ]
          }
        },
        "539a3ea318c4474289bc8d786086e4bf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7ff18e2d227b416ca665fa1f94116b1b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a1ec31b051b047f392429bcec8333cde",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ebe7bb4588bc48fd804c9912f9dfcab9"
          }
        },
        "5120ce6bb16b44918287396bf92d14c1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7cfe220c5ec94ef298a9712e7a611028",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1836118098b7423096abe7bd891dacf5"
          }
        },
        "01645d12521d436d87b92fb6c5e5b3ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9df2aa32dff84bd7ac6e17cebd59b388",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 10000/0 [00:08&lt;00:00, 448.14 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_363641b860ef4739b6ec297d3bffdb23"
          }
        },
        "a1ec31b051b047f392429bcec8333cde": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ebe7bb4588bc48fd804c9912f9dfcab9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7cfe220c5ec94ef298a9712e7a611028": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1836118098b7423096abe7bd891dacf5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": "20px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9df2aa32dff84bd7ac6e17cebd59b388": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "363641b860ef4739b6ec297d3bffdb23": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3877d365f3104973988af5cf1760e83d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_3278172dbcdd4bcbaed94a39259239ac",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d47c6ddeebfb441a8ed17386dcf1cee0",
              "IPY_MODEL_a3b48607072643e2b2b7b74b813baf92",
              "IPY_MODEL_d2d813b173fb4df996b4871e0210f94f"
            ]
          }
        },
        "3278172dbcdd4bcbaed94a39259239ac": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d47c6ddeebfb441a8ed17386dcf1cee0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_dacf2d2eef6e4d0eaa45824edcbe7875",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4c8ed24535d54a8b89beed8c94a4ee9f"
          }
        },
        "a3b48607072643e2b2b7b74b813baf92": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c5567f59790144c4bc5d45af3584cb3b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 2,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 2,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_46a72d3dc8ff4a77a9e6418834cda9d7"
          }
        },
        "d2d813b173fb4df996b4871e0210f94f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e8d2a8ed1fc646a4b337d437ef049165",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2/2 [00:00&lt;00:00, 37.27it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d7a7a3ef0024422984335d8f700e1e5b"
          }
        },
        "dacf2d2eef6e4d0eaa45824edcbe7875": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4c8ed24535d54a8b89beed8c94a4ee9f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c5567f59790144c4bc5d45af3584cb3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "46a72d3dc8ff4a77a9e6418834cda9d7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e8d2a8ed1fc646a4b337d437ef049165": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d7a7a3ef0024422984335d8f700e1e5b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dfcfb25c91d74a14a8223deeb24d9761": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_cb1e07f7035447268a23c34472072f4d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_61e2fd49961e4b22804fd4c3e0f7de74",
              "IPY_MODEL_3e9d292a0dce49938885e1d67105d5b8",
              "IPY_MODEL_f51aeb18d4c24b7cb32aa83a9c4b419a"
            ]
          }
        },
        "cb1e07f7035447268a23c34472072f4d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "61e2fd49961e4b22804fd4c3e0f7de74": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_63e203d772f44488ad05b99024f615de",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_13289a01e5514f248cb08c540c85275c"
          }
        },
        "3e9d292a0dce49938885e1d67105d5b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d5dd7390229043a8871ac4436a1de667",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 47651,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 47651,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b0e589118d3c497c96444e583d25b0e5"
          }
        },
        "f51aeb18d4c24b7cb32aa83a9c4b419a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_34bcec5976c048bcafd6afcaeaaea613",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 46.5k/46.5k [00:00&lt;00:00, 219kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2f4366327ee747e597482858534f666f"
          }
        },
        "63e203d772f44488ad05b99024f615de": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "13289a01e5514f248cb08c540c85275c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d5dd7390229043a8871ac4436a1de667": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b0e589118d3c497c96444e583d25b0e5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "34bcec5976c048bcafd6afcaeaaea613": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2f4366327ee747e597482858534f666f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "519409a56f174eb19689ef0129d81f40": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f46027c84df740b78bf279b220b15aef",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_96e99e9d863c4b29862bb287e2143bb0",
              "IPY_MODEL_70cf1d08ec94443896bed12cafa3ced6",
              "IPY_MODEL_57cd776200534f05a12f17f77a4de2ae"
            ]
          }
        },
        "f46027c84df740b78bf279b220b15aef": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "96e99e9d863c4b29862bb287e2143bb0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8453391148e1490fa333f23701d4c8d2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f6d255bd24e94ad4a3a952acb46134ec"
          }
        },
        "70cf1d08ec94443896bed12cafa3ced6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f5c27b526bf84e5e8cd5b620023c5acb",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 560,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 560,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ef707f7709644203bb18a0942b6a35a1"
          }
        },
        "57cd776200534f05a12f17f77a4de2ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c226e96cd79445f39d3cd55731b7594c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 560/560 [00:00&lt;00:00, 14.8kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0c10105ce85b4428b7755c521e8f3917"
          }
        },
        "8453391148e1490fa333f23701d4c8d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f6d255bd24e94ad4a3a952acb46134ec": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f5c27b526bf84e5e8cd5b620023c5acb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ef707f7709644203bb18a0942b6a35a1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c226e96cd79445f39d3cd55731b7594c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0c10105ce85b4428b7755c521e8f3917": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2ac0627c5d734a0590a24647091ed797": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4b66657839ac4af1b971151284e6a942",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7456c908745c4465804fefeb7f9b6f5b",
              "IPY_MODEL_0ce2ecc617df4e4699d2697f3b4e7121",
              "IPY_MODEL_0054ba5affa24cf29db2b75cc4360ee7"
            ]
          }
        },
        "4b66657839ac4af1b971151284e6a942": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7456c908745c4465804fefeb7f9b6f5b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_87340c53b9f74fcfa7d9741028c0298c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b9c4971b3e0d49f393a7ec16a3ca4df5"
          }
        },
        "0ce2ecc617df4e4699d2697f3b4e7121": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a4f9cfe1a3194dc39172e5a0066b5a2c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 331998265,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 331998265,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_096c13b35e1547b2b7020a38f42f9a7e"
          }
        },
        "0054ba5affa24cf29db2b75cc4360ee7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_745c7dfa2ac345719119d51ae98caa0e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 317M/317M [00:06&lt;00:00, 51.0MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a2235ec98e6146e09859685d1adfdc37"
          }
        },
        "87340c53b9f74fcfa7d9741028c0298c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b9c4971b3e0d49f393a7ec16a3ca4df5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a4f9cfe1a3194dc39172e5a0066b5a2c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "096c13b35e1547b2b7020a38f42f9a7e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "745c7dfa2ac345719119d51ae98caa0e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a2235ec98e6146e09859685d1adfdc37": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "122c39dbac5244c38544e0dc452b4fca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_65d4e09b72ad4595907214ba9503b261",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_9b60dddcd3024f8bb426baa6c3a12a4b",
              "IPY_MODEL_6b966fc093474d1fbedc3f7d031a105c",
              "IPY_MODEL_07af51339b774d52b3bc8a9915d2d73e"
            ]
          }
        },
        "65d4e09b72ad4595907214ba9503b261": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9b60dddcd3024f8bb426baa6c3a12a4b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_73b1aa0b3f9b4531827275f3fcbd1f11",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_77a31fbc0acc4db2898c2d510398456e"
          }
        },
        "6b966fc093474d1fbedc3f7d031a105c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e932299cfc3f4237818704887c60acfe",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 25000,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 25000,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2572917ce5c941c8837b731512104a8e"
          }
        },
        "07af51339b774d52b3bc8a9915d2d73e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_651f325726264c59ae69485231336305",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 25000/25000 [58:11&lt;00:00,  7.18ba/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8be6f18cc33342c69b325bd500fc1d6c"
          }
        },
        "73b1aa0b3f9b4531827275f3fcbd1f11": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "77a31fbc0acc4db2898c2d510398456e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e932299cfc3f4237818704887c60acfe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2572917ce5c941c8837b731512104a8e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "651f325726264c59ae69485231336305": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8be6f18cc33342c69b325bd500fc1d6c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d840b4133b7c434cac7f64e1cfc94f48": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b462c9ab191d42c594ffec9da2a65239",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c2aa83c40e534a6a86a856f925ebe8ef",
              "IPY_MODEL_4024a78bc3db4d4f9604d67a386d256d",
              "IPY_MODEL_0cf9a14a013444de84ee45a328377b33"
            ]
          }
        },
        "b462c9ab191d42c594ffec9da2a65239": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c2aa83c40e534a6a86a856f925ebe8ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4ff1256d8173433985c732fb2862698b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bf1f3e6c1c1a4a5b83d5557dd2d4f4d1"
          }
        },
        "4024a78bc3db4d4f9604d67a386d256d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a68f0478ccf440c3a0c4d9ee86970645",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 5000,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 5000,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b1353b8778bc48a4840dc34a2b31f943"
          }
        },
        "0cf9a14a013444de84ee45a328377b33": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c9f8af6e203947168087c56186634722",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 5000/5000 [11:47&lt;00:00,  6.95ba/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_549e7b058f1f4d02b581d2cf27a7e1f8"
          }
        },
        "4ff1256d8173433985c732fb2862698b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bf1f3e6c1c1a4a5b83d5557dd2d4f4d1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a68f0478ccf440c3a0c4d9ee86970645": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b1353b8778bc48a4840dc34a2b31f943": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c9f8af6e203947168087c56186634722": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "549e7b058f1f4d02b581d2cf27a7e1f8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "69e3a8f28ec84bf095e03778a93422dc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b5f7bcd99e3a4f05bb8f9efd8bf156c2",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2ef30ee33f994cbaab11019c042acd24",
              "IPY_MODEL_0f1938f9c339459290ca04813ded56c2",
              "IPY_MODEL_b9a5dc2d51dc49f592c3b6a1eb16045a"
            ]
          }
        },
        "b5f7bcd99e3a4f05bb8f9efd8bf156c2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2ef30ee33f994cbaab11019c042acd24": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a51d224b38ff432faa2a1e345fd6f725",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_131dfd3ca55a4343b3fa02618df55b2e"
          }
        },
        "0f1938f9c339459290ca04813ded56c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1a72fc51a09a44b6904faf16db81a7a5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 25,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 25,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e184bf3b7e294bd0ab510563b8e6e7a3"
          }
        },
        "b9a5dc2d51dc49f592c3b6a1eb16045a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_cc636bf1c92d4ec3a96b999eeaf3beae",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 25/25 [1:03:32&lt;00:00, 151.35s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6ea28db95943424581e8fa9b5b3dbc3c"
          }
        },
        "a51d224b38ff432faa2a1e345fd6f725": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "131dfd3ca55a4343b3fa02618df55b2e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1a72fc51a09a44b6904faf16db81a7a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e184bf3b7e294bd0ab510563b8e6e7a3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cc636bf1c92d4ec3a96b999eeaf3beae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6ea28db95943424581e8fa9b5b3dbc3c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/ImageGPT/Linear_probing_with_ImageGPT.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3_ZYD8bfKM-c"
      },
      "source": [
        "# Linear probing with ImageGPT\n",
        "\n",
        "In this notebook, we are going to perform \"linear probing\" using a pre-trained ImageGPT. What does that mean?\n",
        "\n",
        "Linear probing means fitting a linear classifier (like logistic regression) on the fixed features of a pre-trained model. In other words, we will only use ImageGPT to produce fixed features X of images, on which we will then fit a linear classifier together with the labels y. The task of interest is image classification.\n",
        "\n",
        "As shown in the [ImageGPT paper](https://cdn.openai.com/papers/Generative_Pretraining_from_Pixels_V2.pdf), fitting a linear classifier on ImageGPT's features results in 96.3% accuracy on CIFAR-10. Also interesting to note is that the best features lie in the middle of ImageGPT's network, rather than at the end. This is probably explained by the fact that ImageGPT's pre-training objective is producing the next pixel, which might not be that helpful for image classification. Hence the model first builds a \"global image representation\" in the middle of the network, which is then used for conditional image generation.\n",
        "\n",
        "Note that the features we are going to use are learned entirely unsupervised (i.e. without any labels). Hence, it's impressive that we can achieve high image classification accuracy using these features.\n",
        "\n",
        "## Set-up environment"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Aw7A3Cg9KJ1S",
        "outputId": "f97f8e00-3deb-49e6-c9d8-cc39425db309"
      },
      "source": [
        "!pip install -q git+https://github.com/huggingface/transformers.git datasets"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[K     |████████████████████████████████| 290 kB 8.4 MB/s \n",
            "\u001b[K     |████████████████████████████████| 596 kB 60.1 MB/s \n",
            "\u001b[K     |████████████████████████████████| 59 kB 9.4 MB/s \n",
            "\u001b[K     |████████████████████████████████| 895 kB 66.2 MB/s \n",
            "\u001b[K     |████████████████████████████████| 3.3 MB 67.8 MB/s \n",
            "\u001b[K     |████████████████████████████████| 243 kB 64.7 MB/s \n",
            "\u001b[K     |████████████████████████████████| 132 kB 77.9 MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.1 MB 61.3 MB/s \n",
            "\u001b[K     |████████████████████████████████| 160 kB 70.7 MB/s \n",
            "\u001b[K     |████████████████████████████████| 192 kB 76.6 MB/s \n",
            "\u001b[K     |████████████████████████████████| 271 kB 73.3 MB/s \n",
            "\u001b[?25h  Building wheel for transformers (PEP 517) ... \u001b[?25l\u001b[?25hdone\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OrAleYN-KWZn"
      },
      "source": [
        "## Download data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 199,
          "referenced_widgets": [
            "8e27071d0c764607b5ca313dd8cf91ea",
            "edb3be86b57d44339d7ded0c46907d79",
            "16231fa674ce46f58e357a4c041c732a",
            "79fc7f3ea9764d11b666e34c3d0b9841",
            "433d6446de20455aaa6cd723dcf15d7f",
            "28de0ea9f2f1492697149582659b46ed",
            "9a14333682a24325a8e44ec1a75b4912",
            "b3c6b3d6c6c34ea4bf33a1fd044d15dc",
            "e8c79f02de81453cbad571b3f025795a",
            "98e3d575d682457b83284d9e06344a54",
            "30a4c61d98f3415bbe2f4a828aa9c446",
            "c1a3e8abfd2044bcac7dda53ecb7e078",
            "63356a99831e4c26a691c91c56c10027",
            "0526b477d4c74cc18e90448f14ecb41f",
            "6169f7d9f0804680bdd331389cd6fb1d",
            "7f73a39a438f4381af9ae318edf6535a",
            "9b1b445b45bc40b98a3255dbdeb21c08",
            "ab5b5c1d7fb647948c0e76a4c9e323df",
            "3244def7ec0444d7a50ed4962448c8f4",
            "1312814b7aa34c16990d910411287537",
            "944bb179e4124334a525621bbde83941",
            "ee18abc5b6f545b3800b2426fabc9057",
            "c51333916bfb45e3b1542cacdee81661",
            "7d31b37ddbfb473580f0bc15d087a214",
            "6de8bd510cfe49aea9e5d8adfccedecf",
            "1955773b8bfc4d1fa7bf5acb10701991",
            "1b2bac60febc49319905da1fea9143ba",
            "c52c0eeb9f70499ebefcf87c7e11087f",
            "9f5237745f434e72a79842062e55eade",
            "033efd3e870f435c88b65f1760e716e1",
            "cfa7811d658b46cbafb47edefcc00e98",
            "730d3f6e486747a7941f25be8eeeb0e0",
            "b2a6c9eecb774c938ebacf80c38cab4b",
            "8e9a09c303784195bd55a227adfa9985",
            "49de16113879490ba222b571a503ad72",
            "2929c29806cc449f81790c61d616724c",
            "cee5a680445b4b599a7e2d5605691917",
            "15e3fec9d0ee43999a7a1ec21fd5d5e6",
            "9e73febb198c4a32a1769eb7fda380f3",
            "a429ada21ede47c9b750b618c66bfa65",
            "a58281f8e21e47f0be708d2555c1bd62",
            "20a73cda113241d79535974214c8e2ec",
            "ac7cf13830094d37813ed65c0cf0ea20",
            "f18f3fc6c4124d54935075f5e45f1f27",
            "f3b96aa457bd498d89c6a4c18bd8695c",
            "539a3ea318c4474289bc8d786086e4bf",
            "7ff18e2d227b416ca665fa1f94116b1b",
            "5120ce6bb16b44918287396bf92d14c1",
            "01645d12521d436d87b92fb6c5e5b3ea",
            "a1ec31b051b047f392429bcec8333cde",
            "ebe7bb4588bc48fd804c9912f9dfcab9",
            "7cfe220c5ec94ef298a9712e7a611028",
            "1836118098b7423096abe7bd891dacf5",
            "9df2aa32dff84bd7ac6e17cebd59b388",
            "363641b860ef4739b6ec297d3bffdb23",
            "3877d365f3104973988af5cf1760e83d",
            "3278172dbcdd4bcbaed94a39259239ac",
            "d47c6ddeebfb441a8ed17386dcf1cee0",
            "a3b48607072643e2b2b7b74b813baf92",
            "d2d813b173fb4df996b4871e0210f94f",
            "dacf2d2eef6e4d0eaa45824edcbe7875",
            "4c8ed24535d54a8b89beed8c94a4ee9f",
            "c5567f59790144c4bc5d45af3584cb3b",
            "46a72d3dc8ff4a77a9e6418834cda9d7",
            "e8d2a8ed1fc646a4b337d437ef049165",
            "d7a7a3ef0024422984335d8f700e1e5b"
          ]
        },
        "id": "IFdVzvCXKMqa",
        "outputId": "c8a1c9ea-9a7a-4bba-cbe9-c981fff677ad"
      },
      "source": [
        "from datasets import load_dataset\n",
        "\n",
        "# #load cifar10 (only small portion for demonstration purposes) \n",
        "# train_ds, test_ds = load_dataset('cifar10', split=['train[:10]', 'test[:10]'])\n",
        "# # split up training into training + validation\n",
        "# splits = train_ds.train_test_split(test_size=0.1)\n",
        "# train_ds = splits['train']\n",
        "# val_ds = splits['test']\n",
        "\n",
        "dataset = load_dataset('cifar10')"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8e27071d0c764607b5ca313dd8cf91ea",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.55k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c1a3e8abfd2044bcac7dda53ecb7e078",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/773 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Downloading and preparing dataset cifar10/plain_text (download: 162.60 MiB, generated: 418.17 MiB, post-processed: Unknown size, total: 580.77 MiB) to /root/.cache/huggingface/datasets/cifar10/plain_text/1.0.0/951e017399fa3b30edfca56468a91ac6a03c9167c0b3f79367e777d1f9a0cf7e...\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c51333916bfb45e3b1542cacdee81661",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/170M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8e9a09c303784195bd55a227adfa9985",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "0 examples [00:00, ? examples/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f3b96aa457bd498d89c6a4c18bd8695c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "0 examples [00:00, ? examples/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Dataset cifar10 downloaded and prepared to /root/.cache/huggingface/datasets/cifar10/plain_text/1.0.0/951e017399fa3b30edfca56468a91ac6a03c9167c0b3f79367e777d1f9a0cf7e. Subsequent calls will reuse this data.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3877d365f3104973988af5cf1760e83d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/2 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FFFDh9zwz3Cw",
        "outputId": "c06bbf7a-6a15-48c3-feb8-2197687e82b4"
      },
      "source": [
        "dataset"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['img', 'label'],\n",
              "        num_rows: 50000\n",
              "    })\n",
              "    test: Dataset({\n",
              "        features: ['img', 'label'],\n",
              "        num_rows: 10000\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GRNrQ0KTKm15"
      },
      "source": [
        "## Load ImageGPT\n",
        "\n",
        "First, let's load ImageGPT as well as its feature extractor (which can be used to prepare images for the model). We'll move the model to GPU for faster inference."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "dfcfb25c91d74a14a8223deeb24d9761",
            "cb1e07f7035447268a23c34472072f4d",
            "61e2fd49961e4b22804fd4c3e0f7de74",
            "3e9d292a0dce49938885e1d67105d5b8",
            "f51aeb18d4c24b7cb32aa83a9c4b419a",
            "63e203d772f44488ad05b99024f615de",
            "13289a01e5514f248cb08c540c85275c",
            "d5dd7390229043a8871ac4436a1de667",
            "b0e589118d3c497c96444e583d25b0e5",
            "34bcec5976c048bcafd6afcaeaaea613",
            "2f4366327ee747e597482858534f666f",
            "519409a56f174eb19689ef0129d81f40",
            "f46027c84df740b78bf279b220b15aef",
            "96e99e9d863c4b29862bb287e2143bb0",
            "70cf1d08ec94443896bed12cafa3ced6",
            "57cd776200534f05a12f17f77a4de2ae",
            "8453391148e1490fa333f23701d4c8d2",
            "f6d255bd24e94ad4a3a952acb46134ec",
            "f5c27b526bf84e5e8cd5b620023c5acb",
            "ef707f7709644203bb18a0942b6a35a1",
            "c226e96cd79445f39d3cd55731b7594c",
            "0c10105ce85b4428b7755c521e8f3917",
            "2ac0627c5d734a0590a24647091ed797",
            "4b66657839ac4af1b971151284e6a942",
            "7456c908745c4465804fefeb7f9b6f5b",
            "0ce2ecc617df4e4699d2697f3b4e7121",
            "0054ba5affa24cf29db2b75cc4360ee7",
            "87340c53b9f74fcfa7d9741028c0298c",
            "b9c4971b3e0d49f393a7ec16a3ca4df5",
            "a4f9cfe1a3194dc39172e5a0066b5a2c",
            "096c13b35e1547b2b7020a38f42f9a7e",
            "745c7dfa2ac345719119d51ae98caa0e",
            "a2235ec98e6146e09859685d1adfdc37"
          ]
        },
        "id": "YsIcGYiNKlZL",
        "outputId": "f6043604-3a35-4f00-a968-3c522708fdc4"
      },
      "source": [
        "from transformers import ImageGPTFeatureExtractor, ImageGPTModel\n",
        "import torch\n",
        "\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "feature_extractor = ImageGPTFeatureExtractor.from_pretrained(\"openai/imagegpt-small\")\n",
        "model = ImageGPTModel.from_pretrained(\"openai/imagegpt-small\")\n",
        "model.to(device)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "dfcfb25c91d74a14a8223deeb24d9761",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/46.5k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "519409a56f174eb19689ef0129d81f40",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/560 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2ac0627c5d734a0590a24647091ed797",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/317M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at openai/imagegpt-small were not used when initializing ImageGPTModel: ['lm_head.weight']\n",
            "- This IS expected if you are initializing ImageGPTModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing ImageGPTModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "ImageGPTModel(\n",
              "  (wte): Embedding(513, 512)\n",
              "  (wpe): Embedding(1024, 512)\n",
              "  (drop): Dropout(p=0.1, inplace=False)\n",
              "  (h): ModuleList(\n",
              "    (0): ImageGPTBlock(\n",
              "      (ln_1): ImageGPTLayerNorm()\n",
              "      (attn): ImageGPTAttention(\n",
              "        (c_attn): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "        (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (ln_2): ImageGPTLayerNorm()\n",
              "      (mlp): ImageGPTMLP(\n",
              "        (c_fc): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "    )\n",
              "    (1): ImageGPTBlock(\n",
              "      (ln_1): ImageGPTLayerNorm()\n",
              "      (attn): ImageGPTAttention(\n",
              "        (c_attn): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "        (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (ln_2): ImageGPTLayerNorm()\n",
              "      (mlp): ImageGPTMLP(\n",
              "        (c_fc): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "    )\n",
              "    (2): ImageGPTBlock(\n",
              "      (ln_1): ImageGPTLayerNorm()\n",
              "      (attn): ImageGPTAttention(\n",
              "        (c_attn): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "        (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (ln_2): ImageGPTLayerNorm()\n",
              "      (mlp): ImageGPTMLP(\n",
              "        (c_fc): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "    )\n",
              "    (3): ImageGPTBlock(\n",
              "      (ln_1): ImageGPTLayerNorm()\n",
              "      (attn): ImageGPTAttention(\n",
              "        (c_attn): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "        (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (ln_2): ImageGPTLayerNorm()\n",
              "      (mlp): ImageGPTMLP(\n",
              "        (c_fc): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "    )\n",
              "    (4): ImageGPTBlock(\n",
              "      (ln_1): ImageGPTLayerNorm()\n",
              "      (attn): ImageGPTAttention(\n",
              "        (c_attn): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "        (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (ln_2): ImageGPTLayerNorm()\n",
              "      (mlp): ImageGPTMLP(\n",
              "        (c_fc): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "    )\n",
              "    (5): ImageGPTBlock(\n",
              "      (ln_1): ImageGPTLayerNorm()\n",
              "      (attn): ImageGPTAttention(\n",
              "        (c_attn): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "        (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (ln_2): ImageGPTLayerNorm()\n",
              "      (mlp): ImageGPTMLP(\n",
              "        (c_fc): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "    )\n",
              "    (6): ImageGPTBlock(\n",
              "      (ln_1): ImageGPTLayerNorm()\n",
              "      (attn): ImageGPTAttention(\n",
              "        (c_attn): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "        (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (ln_2): ImageGPTLayerNorm()\n",
              "      (mlp): ImageGPTMLP(\n",
              "        (c_fc): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "    )\n",
              "    (7): ImageGPTBlock(\n",
              "      (ln_1): ImageGPTLayerNorm()\n",
              "      (attn): ImageGPTAttention(\n",
              "        (c_attn): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "        (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (ln_2): ImageGPTLayerNorm()\n",
              "      (mlp): ImageGPTMLP(\n",
              "        (c_fc): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "    )\n",
              "    (8): ImageGPTBlock(\n",
              "      (ln_1): ImageGPTLayerNorm()\n",
              "      (attn): ImageGPTAttention(\n",
              "        (c_attn): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "        (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (ln_2): ImageGPTLayerNorm()\n",
              "      (mlp): ImageGPTMLP(\n",
              "        (c_fc): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "    )\n",
              "    (9): ImageGPTBlock(\n",
              "      (ln_1): ImageGPTLayerNorm()\n",
              "      (attn): ImageGPTAttention(\n",
              "        (c_attn): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "        (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (ln_2): ImageGPTLayerNorm()\n",
              "      (mlp): ImageGPTMLP(\n",
              "        (c_fc): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "    )\n",
              "    (10): ImageGPTBlock(\n",
              "      (ln_1): ImageGPTLayerNorm()\n",
              "      (attn): ImageGPTAttention(\n",
              "        (c_attn): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "        (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (ln_2): ImageGPTLayerNorm()\n",
              "      (mlp): ImageGPTMLP(\n",
              "        (c_fc): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "    )\n",
              "    (11): ImageGPTBlock(\n",
              "      (ln_1): ImageGPTLayerNorm()\n",
              "      (attn): ImageGPTAttention(\n",
              "        (c_attn): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "        (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (ln_2): ImageGPTLayerNorm()\n",
              "      (mlp): ImageGPTMLP(\n",
              "        (c_fc): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "    )\n",
              "    (12): ImageGPTBlock(\n",
              "      (ln_1): ImageGPTLayerNorm()\n",
              "      (attn): ImageGPTAttention(\n",
              "        (c_attn): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "        (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (ln_2): ImageGPTLayerNorm()\n",
              "      (mlp): ImageGPTMLP(\n",
              "        (c_fc): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "    )\n",
              "    (13): ImageGPTBlock(\n",
              "      (ln_1): ImageGPTLayerNorm()\n",
              "      (attn): ImageGPTAttention(\n",
              "        (c_attn): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "        (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (ln_2): ImageGPTLayerNorm()\n",
              "      (mlp): ImageGPTMLP(\n",
              "        (c_fc): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "    )\n",
              "    (14): ImageGPTBlock(\n",
              "      (ln_1): ImageGPTLayerNorm()\n",
              "      (attn): ImageGPTAttention(\n",
              "        (c_attn): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "        (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (ln_2): ImageGPTLayerNorm()\n",
              "      (mlp): ImageGPTMLP(\n",
              "        (c_fc): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "    )\n",
              "    (15): ImageGPTBlock(\n",
              "      (ln_1): ImageGPTLayerNorm()\n",
              "      (attn): ImageGPTAttention(\n",
              "        (c_attn): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "        (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (ln_2): ImageGPTLayerNorm()\n",
              "      (mlp): ImageGPTMLP(\n",
              "        (c_fc): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "    )\n",
              "    (16): ImageGPTBlock(\n",
              "      (ln_1): ImageGPTLayerNorm()\n",
              "      (attn): ImageGPTAttention(\n",
              "        (c_attn): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "        (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (ln_2): ImageGPTLayerNorm()\n",
              "      (mlp): ImageGPTMLP(\n",
              "        (c_fc): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "    )\n",
              "    (17): ImageGPTBlock(\n",
              "      (ln_1): ImageGPTLayerNorm()\n",
              "      (attn): ImageGPTAttention(\n",
              "        (c_attn): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "        (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (ln_2): ImageGPTLayerNorm()\n",
              "      (mlp): ImageGPTMLP(\n",
              "        (c_fc): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "    )\n",
              "    (18): ImageGPTBlock(\n",
              "      (ln_1): ImageGPTLayerNorm()\n",
              "      (attn): ImageGPTAttention(\n",
              "        (c_attn): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "        (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (ln_2): ImageGPTLayerNorm()\n",
              "      (mlp): ImageGPTMLP(\n",
              "        (c_fc): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "    )\n",
              "    (19): ImageGPTBlock(\n",
              "      (ln_1): ImageGPTLayerNorm()\n",
              "      (attn): ImageGPTAttention(\n",
              "        (c_attn): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "        (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (ln_2): ImageGPTLayerNorm()\n",
              "      (mlp): ImageGPTMLP(\n",
              "        (c_fc): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "    )\n",
              "    (20): ImageGPTBlock(\n",
              "      (ln_1): ImageGPTLayerNorm()\n",
              "      (attn): ImageGPTAttention(\n",
              "        (c_attn): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "        (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (ln_2): ImageGPTLayerNorm()\n",
              "      (mlp): ImageGPTMLP(\n",
              "        (c_fc): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "    )\n",
              "    (21): ImageGPTBlock(\n",
              "      (ln_1): ImageGPTLayerNorm()\n",
              "      (attn): ImageGPTAttention(\n",
              "        (c_attn): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "        (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (ln_2): ImageGPTLayerNorm()\n",
              "      (mlp): ImageGPTMLP(\n",
              "        (c_fc): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "    )\n",
              "    (22): ImageGPTBlock(\n",
              "      (ln_1): ImageGPTLayerNorm()\n",
              "      (attn): ImageGPTAttention(\n",
              "        (c_attn): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "        (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (ln_2): ImageGPTLayerNorm()\n",
              "      (mlp): ImageGPTMLP(\n",
              "        (c_fc): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "    )\n",
              "    (23): ImageGPTBlock(\n",
              "      (ln_1): ImageGPTLayerNorm()\n",
              "      (attn): ImageGPTAttention(\n",
              "        (c_attn): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "        (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (ln_2): ImageGPTLayerNorm()\n",
              "      (mlp): ImageGPTMLP(\n",
              "        (c_fc): Conv1D()\n",
              "        (c_proj): Conv1D()\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (ln_f): ImageGPTLayerNorm()\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LX0ddbEuWSWK"
      },
      "source": [
        "## Preprocess data\n",
        "\n",
        "Let's show how we can obtain a fixed-size feature vector for a single image. This involves 2 things: 1) use ImageGPTFeatureExtractor to prepare the image for the ImageGPT model 2) extract the hidden states of a particular layer, averaged along the sequence dimension. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q4vLGJ7tLboj"
      },
      "source": [
        "from PIL import Image\n",
        "import requests\n",
        "\n",
        "url = 'http://images.cocodataset.org/val2017/000000039769.jpg'\n",
        "image = Image.open(requests.get(url, stream=True).raw)\n",
        "\n",
        "encoding = feature_extractor(image, return_tensors=\"pt\")\n",
        "pixel_values = encoding.pixel_values.to(device)\n",
        "print(pixel_values.shape)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xGLyKVLoL7i0",
        "outputId": "6bf57544-736e-4074-a825-5e4c3b7e2f32"
      },
      "source": [
        "# forward pass\n",
        "outputs = model(pixel_values, output_hidden_states=True)\n",
        "hidden_states = outputs.hidden_states\n",
        "print(len(hidden_states))"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "25\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Aj07fIC-4qyQ"
      },
      "source": [
        "Here, we take the hidden states from the middle of the network (which, according to the paper, results in the best image classification performance), and average them across the sequence (also called time) dimension. This is just a vector of shape `(batch_size, hidden_size)` - the hidden size for the small ImageGPT model is 512."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zINmZoXdvSpz",
        "outputId": "93925b64-b810-419f-85ee-13f733c2d56b"
      },
      "source": [
        "feature_vector = torch.mean(hidden_states[13], dim=1)\n",
        "feature_vector.shape"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([1, 512])"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ALqBx2eoWVn1"
      },
      "source": [
        "Now that we've shown how to do this for a single image, et's now do this for the entire dataset. We can do this fast using HuggingFace Datasets' `.map(function, batched=True)` functionality. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FjjC6zeqK1bA"
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "def extract_features(examples):\n",
        "  # take a batch of images\n",
        "  images = examples['img']\n",
        "  # convert to list of NumPy arrays of shape (C, H, W)\n",
        "  images = [np.array(image, dtype=np.uint8) for image in images]\n",
        "  images = [np.moveaxis(image, source=-1, destination=0) for image in images]\n",
        "  # tokenize images\n",
        "  encoding = feature_extractor(images=images, return_tensors=\"pt\")\n",
        "  pixel_values = encoding.pixel_values.to(device)\n",
        "  # forward through model to get hidden states\n",
        "  with torch.no_grad():\n",
        "    outputs = model(pixel_values, output_hidden_states=True)\n",
        "  hidden_states = outputs.hidden_states\n",
        "  # add features of each layer\n",
        "  for i in range(len(hidden_states)):\n",
        "      features = torch.mean(hidden_states[i], dim=1)\n",
        "      examples[f'features_{i}'] = features.cpu().detach().numpy()\n",
        "  \n",
        "  return examples"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vGXuH4jxdLjj"
      },
      "source": [
        "We do this for both the training and test set:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81,
          "referenced_widgets": [
            "122c39dbac5244c38544e0dc452b4fca",
            "65d4e09b72ad4595907214ba9503b261",
            "9b60dddcd3024f8bb426baa6c3a12a4b",
            "6b966fc093474d1fbedc3f7d031a105c",
            "07af51339b774d52b3bc8a9915d2d73e",
            "73b1aa0b3f9b4531827275f3fcbd1f11",
            "77a31fbc0acc4db2898c2d510398456e",
            "e932299cfc3f4237818704887c60acfe",
            "2572917ce5c941c8837b731512104a8e",
            "651f325726264c59ae69485231336305",
            "8be6f18cc33342c69b325bd500fc1d6c",
            "d840b4133b7c434cac7f64e1cfc94f48",
            "b462c9ab191d42c594ffec9da2a65239",
            "c2aa83c40e534a6a86a856f925ebe8ef",
            "4024a78bc3db4d4f9604d67a386d256d",
            "0cf9a14a013444de84ee45a328377b33",
            "4ff1256d8173433985c732fb2862698b",
            "bf1f3e6c1c1a4a5b83d5557dd2d4f4d1",
            "a68f0478ccf440c3a0c4d9ee86970645",
            "b1353b8778bc48a4840dc34a2b31f943",
            "c9f8af6e203947168087c56186634722",
            "549e7b058f1f4d02b581d2cf27a7e1f8"
          ]
        },
        "id": "UqOIk1abNW-n",
        "outputId": "6a8dfc87-0a95-4002-9816-422c2c2c6884"
      },
      "source": [
        "encoded_dataset = dataset.map(extract_features, batched=True, batch_size=2)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "122c39dbac5244c38544e0dc452b4fca",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/25000 [00:00<?, ?ba/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d840b4133b7c434cac7f64e1cfc94f48",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/5000 [00:00<?, ?ba/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NRDT0gRn3zMO"
      },
      "source": [
        "We add the following line for a speed-up:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gNG8M69oUBDP"
      },
      "source": [
        "encoded_dataset = encoded_dataset.with_format(\"numpy\")"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0ekFYALY4lED"
      },
      "source": [
        "We can save the processed dataset to disk as follows:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ut7jxAQgJK_T"
      },
      "source": [
        "encoded_dataset.save_to_disk(\"/content/drive/MyDrive/ImageGPT\")"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZcU4gVf7R3Mz"
      },
      "source": [
        "## Train sklearn model\n",
        "\n",
        "Next, we can fit a logistic regression model on the (X, y) pairs. We do this for the features X of each layer of the Transformer decoder.\n",
        "\n",
        "Next, we evaluate the fitted model on both the training and validation set of CIFAR10, and store the result in a dictionary."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3HoQxyF2SL9e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "355abc8d-f052-4e81-be46-4457bc2a3cc4"
      },
      "source": [
        "encoded_dataset['train']"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Dataset({\n",
              "    features: ['features_0', 'features_1', 'features_10', 'features_11', 'features_12', 'features_13', 'features_14', 'features_15', 'features_16', 'features_17', 'features_18', 'features_19', 'features_2', 'features_20', 'features_21', 'features_22', 'features_23', 'features_24', 'features_3', 'features_4', 'features_5', 'features_6', 'features_7', 'features_8', 'features_9', 'img', 'label'],\n",
              "    num_rows: 50000\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u1NXa0QbVEZG",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2fc37d47-daae-40dc-9f29-6f257426eaa2"
      },
      "source": [
        "encoded_dataset['train']['features_0'][0].shape"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(512,)"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ilDb2Z9ESNd0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bdb136bf-48b8-4845-f4ee-9030776c1bbc"
      },
      "source": [
        "encoded_dataset['train']['label']"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([6, 9, 9, ..., 9, 1, 1])"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d5PJCscNRV6z",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "69e3a8f28ec84bf095e03778a93422dc",
            "b5f7bcd99e3a4f05bb8f9efd8bf156c2",
            "2ef30ee33f994cbaab11019c042acd24",
            "0f1938f9c339459290ca04813ded56c2",
            "b9a5dc2d51dc49f592c3b6a1eb16045a",
            "a51d224b38ff432faa2a1e345fd6f725",
            "131dfd3ca55a4343b3fa02618df55b2e",
            "1a72fc51a09a44b6904faf16db81a7a5",
            "e184bf3b7e294bd0ab510563b8e6e7a3",
            "cc636bf1c92d4ec3a96b999eeaf3beae",
            "6ea28db95943424581e8fa9b5b3dbc3c"
          ]
        },
        "outputId": "392371ce-6830-4e45-ddfa-0fb8d3861eb4"
      },
      "source": [
        "from sklearn.linear_model import LogisticRegression\n",
        "from tqdm.notebook import tqdm\n",
        "\n",
        "train_dataset = encoded_dataset['train']\n",
        "test_dataset = encoded_dataset['test']\n",
        "\n",
        "scores = dict()\n",
        "for i in tqdm(range(model.config.n_layer + 1)):\n",
        "    # fit linear classifier\n",
        "    lr_clf = LogisticRegression(max_iter=1000)\n",
        "    lr_clf.fit(train_dataset[f'features_{i}'], train_dataset['label'])\n",
        "    # compute accuracy on training + test set\n",
        "    training_score = lr_clf.score(train_dataset[f'features_{i}'], train_dataset['label'])\n",
        "    test_score = lr_clf.score(test_dataset[f'features_{i}'], test_dataset['label'])\n",
        "    scores[f'features_{i}'] = (training_score, test_score)"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "69e3a8f28ec84bf095e03778a93422dc",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/25 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sG7LiTzi5Qjw"
      },
      "source": [
        "Running the cell above can take a while. I've stored the dictionary as seen below:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JiMDbovQb4xG"
      },
      "source": [
        "scores = {'features_0': (0.35678, 0.3328),\n",
        " 'features_1': (0.39662, 0.3979),\n",
        " 'features_10': (0.75778, 0.7556),\n",
        " 'features_11': (0.76852, 0.764),\n",
        " 'features_12': (0.78118, 0.7777),\n",
        " 'features_13': (0.79084, 0.784),\n",
        " 'features_14': (0.79096, 0.7823),\n",
        " 'features_15': (0.78704, 0.7828),\n",
        " 'features_16': (0.79426, 0.7862),\n",
        " 'features_17': (0.77646, 0.7679),\n",
        " 'features_18': (0.79218, 0.7823),\n",
        " 'features_19': (0.77898, 0.7686),\n",
        " 'features_2': (0.4538, 0.4555),\n",
        " 'features_20': (0.76906, 0.758),\n",
        " 'features_21': (0.7632, 0.7477),\n",
        " 'features_22': (0.76064, 0.7422),\n",
        " 'features_23': (0.7458, 0.7238),\n",
        " 'features_24': (0.69894, 0.6775),\n",
        " 'features_3': (0.47694, 0.4739),\n",
        " 'features_4': (0.49574, 0.4954),\n",
        " 'features_5': (0.52214, 0.5229),\n",
        " 'features_6': (0.55212, 0.5503),\n",
        " 'features_7': (0.5959, 0.5887),\n",
        " 'features_8': (0.65802, 0.6532),\n",
        " 'features_9': (0.68232, 0.6791)}"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kH9nnoMJbaqr"
      },
      "source": [
        "Let's plot this (as was done in the paper). We plot the index of each layer on the horizontal axis, and the accuracy on the test set of CIFAR10 on the vertical axis."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "W23EQeqg0lGU",
        "outputId": "de471b29-e46a-4dc3-fd4c-80697dc87b8b"
      },
      "source": [
        "# turn keys into integers which we can sort\n",
        "values = {int(k.replace('features_', '')):v for k, v in scores.items()}\n",
        "values = dict(sorted(values.items()))\n",
        "values"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{0: (0.35678, 0.3328),\n",
              " 1: (0.39662, 0.3979),\n",
              " 2: (0.4538, 0.4555),\n",
              " 3: (0.47694, 0.4739),\n",
              " 4: (0.49574, 0.4954),\n",
              " 5: (0.52214, 0.5229),\n",
              " 6: (0.55212, 0.5503),\n",
              " 7: (0.5959, 0.5887),\n",
              " 8: (0.65802, 0.6532),\n",
              " 9: (0.68232, 0.6791),\n",
              " 10: (0.75778, 0.7556),\n",
              " 11: (0.76852, 0.764),\n",
              " 12: (0.78118, 0.7777),\n",
              " 13: (0.79084, 0.784),\n",
              " 14: (0.79096, 0.7823),\n",
              " 15: (0.78704, 0.7828),\n",
              " 16: (0.79426, 0.7862),\n",
              " 17: (0.77646, 0.7679),\n",
              " 18: (0.79218, 0.7823),\n",
              " 19: (0.77898, 0.7686),\n",
              " 20: (0.76906, 0.758),\n",
              " 21: (0.7632, 0.7477),\n",
              " 22: (0.76064, 0.7422),\n",
              " 23: (0.7458, 0.7238),\n",
              " 24: (0.69894, 0.6775)}"
            ]
          },
          "metadata": {},
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 282
        },
        "id": "ky1ycTr8bcik",
        "outputId": "87d75380-c78b-4575-d5cc-a76740633dde"
      },
      "source": [
        "%matplotlib inline\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "fig = plt.figure()\n",
        "ax = plt.axes()\n",
        "x = list(values.keys())\n",
        "y = [value[1][1] for value in values.items()]\n",
        "ax.plot(x, y)"
      ],
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7f6d78a11a10>]"
            ]
          },
          "metadata": {},
          "execution_count": 49
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXhV5b328e8vM0NCgIRIQiABwhCQMaIIKg5YhwrOhWqVWtQODp3Oe6zt26rn2On12Ho86qlSHKsIikgtikNFURRIIAIJMwQyEAhjgiHjft4/Em1EIAkkWXu4P9fFleyVlex7scztw7Mmc84hIiLBJczrACIi0vZU7iIiQUjlLiIShFTuIiJBSOUuIhKEIrx644SEBJeWlubV24uIBKScnJy9zrnE5tZrUbmb2SXAI0A4MMs59/ujvt4XeBaIb1znHufcohP9zLS0NLKzs1vy9iIi0sjMdrRkvWanZcwsHHgMuBTIBKabWeZRq/0KmOucGw1MAx5vXVwREWlLLZlzHwdscc5tc87VAHOAqUet44C4xs+7ASVtF1FERFqrJeWeAhQ2eV3UuKyp+4AbzawIWATceawfZGa3mVm2mWWXlZWdRFwREWmJtjpbZjrwjHOuD3AZ8LyZfe1nO+eedM5lOeeyEhObPR4gIiInqSXlXgykNnndp3FZU98D5gI45z4BYoCEtggoIiKt15JyXwlkmFm6mUXRcMB04VHr7AQuBDCzoTSUu+ZdREQ80my5O+fqgDuAxcB6Gs6KyTOzB8xsSuNqPwNuNbPPgJeAGU63mxQR8UyLznNvPGd90VHLft3k83xgQttGE5HWcM6x//MaSg5WUXLoCCUHj9AlKoKrx6QQEa6L0UONZ1eoigS6mjof+bvKyd15gN0V1XSNjiAuJoLYmEhiv/Kx4fOu0RGEh9lJvZfP56iu81FaXkXJwSMUH2wo74Y//1pWXef72vc+vayA3119OqNS4091kyWAqNxFWsA5R9GBI6wuPMjqnQfILTxIXkk5NY1lGhFm1Pman4nsGh3xZeGHWcP31PsctfW+xo+Oep+PunpHnc9R5/NR53Mcb5KzV2w0yfGdGNo7jguH9iI5vhPJ8Z1Iafy4Yvt+frNwHVc9/jE3j0/jZxcPIjYmsk3+TurqfbyeW8LCz0qYODCB6Wf2pWu0KsVfmFdT41lZWU63HxB/VVFVy5qiQ+Q2KfO9h2sAiIkM4/SUbozu251RqfGM7htP726dqKqtp6Kqjoqq2saP//q8/BjLfM4RGR5GeJgREWZEhBvhYWFEhhvhYfbVr4WFERlhJMXGfFneSd2iiY4Ib9G2PLR4I899uoOk2BjunzqMbww77aT/bmrrfby2upjH3t/Cjn2V9IqNZk9FNd06RXLz+H7MmJBOjy5RJ/3z5cTMLMc5l9Xseip3kQZHaup5eeVO5qwsZOPuii9Hy/0TuzA6tTuj+sYzOjWewafFEhmAc9irdh7g3vlr2VBawcWZSdw/dRi9u3Vq8ffX1vuYv6qI/3l/C4X7jzA8JY67LshgcmYSuYUHeWLJVt7O302nyHCmjUtl5jn9SYlv+c+XllG5i7TQocpanvukgKeXFbD/8xpG941n0qBejOobz6g+8XTr3DbTGP6gtt7HrKXbeeS9TUSEhfHziwfxnfFpJzwWUFPn45WcIh57fwvFB48wok837r4wgwuG9MLsq9+3eXcF//vBNl7PbbgU5srRKXz/vP4M7BXbrtsVSlTuIs3YU17FrI+287dPd/B5TT3nD07kh+cP5Iy0Hl5Ha3c791XyywVrWbp5LyNT4/ndVaeTmRz3lXWq6+qZl13EE0u2UnzwCCNT4/nxhRlMGpz4tVI/WtGBSmYt3c6clTuprvNxcWYSP5w0kJE6qHvKVO4ix1Gw93P+8uE2Xs0pos7n45sjkvn+eQO+Vm7BzjnHws9KeODv+Rw8UsvMiencfVEGYWbMzS7kiSVb2XWoijF947n7okGcm5HQbKkfbd/hap5dVsAzywoor6rj7AE9+eGkgUwY2LPVP0saqNxFjpJXcognlmxl0dpdRISHcd3YPtx2bn/69ezidTRPHays4XeLNvBydiF9uneitt7H7vJqsvp15+6LMpg4sPWlfrTD1XW8tHwnsz7axu7yaoYlx3Ht2D5cMTKZhK7RbbQloUHlLkLD6HTF9v08vmQrH2wqo2t0BDec1ZfvTUinV1yM1/H8yvJt+/jtovV0jorgzgsGMn5A24+uq+vqmb+qmBc+3UFeSTnhYcakQYlcNSaFi4YmERPZ/Nk/oU7lLiHL53PklZTz/sY9vJO/m7XFh+jZJYpbJqZz41n96NYpeA6QBrKNpRXMX13E66tLKC2vIjY6gstH9Oaq0SmckdaDsJO84CvYqdwlpBw6UstHm/fy/sY9LNlYxt7D1ZjBiJRuXDO2D9dnpWpU6KfqfY5Ptu5j/uoi3lpXSmVNPSnxnbh6TApXjU6hf2JXryP6FZW7BDXnHBtKK1iysYz3N+4hZ8cB6n2Obp0iOXdQIucPTuTcQYmazw0wlTV1vJ23m1dXFfHxlr34HIxMjefq0SlcMTJZF0ehcpcgVF1Xz5KNZSzZuIf3N5RRWl4FQGbvOM4fksj5g3sxKjVeN8kKEnvKq3g9t4T5q4tZv6uciDDj3EGJTB2VzOTMJDpHheatDlTuEnR+OjeX+auK6RodwcSBCZw/JJHzBvXitG46MBrs1u8qZ0FuMX/PLaHkUBWdo8K5ODOJqaNSmJiREJBXDJ8slbsElYOVNYx78D2uGp3Cf1w5nKiI0Pllln/x+RwrC/azILeERWt3cehILT26RHH56b25cnQyY/p2D/rz51ta7qH57xoJOK/nllBT7+Oms/up2ENYWJhxZv+enNm/J/dPGcYHm8pYkFvM3OxCnv90B326d2LqqGSuHJVCRlJo3/JA5S4BYW52IcOS4xiW3M3rKOInoiLCmJyZxOTMJA5X17F4XSkLcot5YslWHnt/K0NOi+Ws/j3J7B1HZnIcGUldW3QXzWChche/l19STl5JOfddkel1FPFTXaMjuGZsH64Z24eyimreWFPCm2tLmZtdSGVNPQCR4cbAXrFk9o5jWHJD4WcmxxHXRve39zcqd/F783IKiQoPY+qoFK+jSABIjI3muxPS+e6EdHw+R8G+z8nf1TBAyC8p54NNZby6qujL9VN7dGJY725kJseRldad8f2D4743KnfxazV1PhasLmZyZhLddY6ztFJYmNE/sSv9E7vyzRHJXy7fU1H15b8I83c1lP5beaUAXDikFw9cOTzg70Wvche/9t763RyorOXarD5eR5Eg0is2hl6DY5g0uNeXy764udnD72xi8sMf8POLB3Pz2Se+170/02kH4tfm5RRxWlwM52Ykeh1FglzX6AhuPbc/b//kXMal9+CBN/K5+vGPySs55HW0k6JyF7+1u7yKJRv3cPWYlIAdPUngSe3RmadnnMF/Tx9N8cEjTPmfj/ndm+s50nhgNlCo3MVvzV9VjM/BtWM1JSMdy8yYMjKZd396HteO6cNfPtjGxX/+gA83lXkdrcVU7uKXnHPMyynkjLTuuiugeCa+cxR/uHYEc247i8iwMG6avYKfvJzLvsPVXkdrlspd/NKqnQfYVvY5141N9TqKCGf178miu8/hrgsG8saaEi58+ANeySnCq9u3tITKXfzSvOwiOkWGc9mI3l5HEQEgJjKcn148mEV3ncPAxK78fN5n3DBrOTv3VXod7ZhU7uJ3KmvqeGPNLi4f0Zuu0TpbV/xLRlIsc28fz4NXDWdt0SG+M3s5VbX+d7BV5S5+5821pRyuruM6HUgVPxUWZtxwZj/+8p2x7NhXyaP/3Ox1pK9RuYvfmZdTSFrPzoxL7+F1FJETOntgAtc0nk2zsbTC6zhfoXIXv7JzXyWfbtvPtWP7BMX9PST4/fLyocTGRHDva2vx+fznAKvKXfzKKzmFmMHVYzQlI4GhR5cofnl5Jjk7DvDiip1ex/mSyl38hs/neHVVMedkJJIc4DdtktByzZgUzh7Qkz+8tYE9jc/29ZrKXfzGsq37KD54RAdSJeCYGQ9edTrVdT7ufyPf6ziAyl38yNzsQuJiIpicmeR1FJFWS0/owp3nD+Qfa3bx/oY9XsdpWbmb2SVmttHMtpjZPcf4+p/MLLfxzyYzO9j2USWYHTpSy+K8Uq4cnUJMZOg8Ck2Cy+3nDSCjV1d+tWAdlTV1nmZpttzNLBx4DLgUyASmm9lXnnfmnPuJc26Uc24U8Cgwvz3CSvD6+2clVNf5dLsBCWhREWH89urTKT54hD+9s8nTLC0ZuY8DtjjntjnnaoA5wNQTrD8deKktwknomJddyJDTYhmeEud1FJFTckZaD6aP68vsjwtYV+zdveBbUu4pQGGT10WNy77GzPoB6cA/j/P128ws28yyy8oC59aZ0r427a7gs6JDXJeVqnPbJSjcc8kQuneO4t7X1lLv0bnvbX1AdRrwinPumDdacM496ZzLcs5lJSbqyTrSYF52IRFhxpWjkptfWSQAdOscya+vyGRN0SGe+6TAkwwtKfdioOlEaJ/GZccyDU3JSCvU1vt4bXUxFw1NomfXaK/jiLSZK0b05rxBiTy0eCMlB490+Pu3pNxXAhlmlm5mUTQU+MKjVzKzIUB34JO2jSjB7P0Ne9h7uIbr9ABsCTJmxn9eOZx65/jNwrwOf/9my905VwfcASwG1gNznXN5ZvaAmU1psuo0YI7z57vXi9+Zl1NEYmw05w3SNJ0En9QenfnJRYN4J383b60r7dD3btHNsp1zi4BFRy379VGv72u7WBIKyiqq+eeGPcw8J52IcF1PJ8HplonpLMgt4b6FeUwY2JPYmMgOeV/9RolnFqwupt7ndG67BLXI8DB+d/Xp7K6o4qHFGzvsfVXu4gnnHHOzCxndN56BvfQAbAluo1Ljuemsfjz36Q5W7zzQIe+pchdPvLd+D5v3HOb6LI3aJTT8/BuDSYqN4Rfz11Jb72v391O5S4d7bXUR338hhyGnxTJlpM5tl9AQGxPJfVOGsaG0gmeXFbT7++npw9JhnHP85cNt/P7NDYzv35O/3DSWLnoAtoSQS4afxv1ThnXIoEa/WdIh6n2O/3gjn2eWFXDFyGQeum4E0RG6+6OEnpvPTuuQ91G5S7urqq3nJy/n8ua6Um49J51fXDqUsDDdQ0akPancpV0drKzhtudyWFGwn19dPpSZ5/T3OpJISFC5S7spPniEm2evYOe+Sh6dPpordPBUpMOo3KVdrN9VzoynV1BZU8+zt4xj/ICeXkcSCSkqd2lzy7bs5fbnc+gSHcG8749nyGl6AIdIR1O5S5ta+FkJP5ubS3pCF5757jiS4zt5HUkkJKncpc089eE2Hly0nnHpPXjqO1l069wxN0gSka9TuUub+O2i9Tz54TYuP703/3X9SGIidQ67iJdU7nLK1hQd5MkPtzF9XCoPXnm6zmEX8QO6t4ycsqeWbic2OoJ7L9PFSSL+QuUup6ToQCWL1u5i2rjUDnsIgYg0T+Uup+TpjwsAmDEh3dsgIvIVKnc5aYeO1DJnxU6+OaI3KTrlUcSvqNzlpM1ZsZPPa+q5VfeLEfE7Knc5KTV1Pp7+uIDx/XsyPKWb13FE5Cgqdzkp/1hbQml5Fbeeq7l2EX+kcpdWc87x1IfbGdirK5MG9fI6jogcg8pdWu2TrfvI31XOzInpOq9dxE+p3KXVnly6jYSuUVw5OsXrKCJyHCp3aZVNuytYsrGMm8an6f4xIn5M5S6tMmvpNmIiw7jxrH5eRxGRE1C5S4vtqahiweoSrh3bhx5doryOIyInoHKXFntu2Q5qfT6+N1EXLYn4O5W7tEhlTR0vLN/B5KFJpCd08TqOiDRD5S4t8mpOEQcra7ntXI3aRQKByl2aVe9zzPpoO6NS4xnbr7vXcUSkBVTu0qx38nezY18lt57THzNdtCQSCFTu0qynlm4jtUcnvjEsyesoItJCLSp3M7vEzDaa2RYzu+c461xvZvlmlmdmL7ZtTPFKzo4D5Ow4wC0T0okI11hAJFA0+4BsMwsHHgMmA0XASjNb6JzLb7JOBvALYIJz7oCZ6W5SQWLW0m3ExURwfVaq11FEpBVaMhQbB2xxzm1zztUAc4CpR61zK/CYc+4AgHNuT9vGFC/s3FfJ4rxSbjirH12imx0HiIgfaUm5pwCFTV4XNS5rahAwyMw+NrNPzeySY/0gM7vNzLLNLLusrOzkEkuHmf3xdsLDjBlnp3kdRURaqa0mUSOADGASMB14yszij17JOfekcy7LOZeVmJjYRm8t7eFgZQ0vryxkysgUkuJivI4jIq3UknIvBppOuPZpXNZUEbDQOVfrnNsObKKh7CVA/W35To7U1utJSyIBqiXlvhLIMLN0M4sCpgELj1pnAQ2jdswsgYZpmm1tmFM6UHVdPc8sK+CcjASGnBbndRwROQnNlrtzrg64A1gMrAfmOufyzOwBM5vSuNpiYJ+Z5QPvA//mnNvXXqGlfS3MLaGsolq3GhAJYC06BcI5twhYdNSyXzf53AE/bfwjAaysopo/Lt5IZu84Jg5M8DqOiJwknd8mX6r3Oe6es5qKqlqe/9443WpAJICp3OVLj7y3mWVb9/H/rh2huXaRAKfryQWADzeV8eg/N3Pd2D5cp6tRRQKeyl0oPVTFj1/OZVCvWB6YOtzrOCLSBlTuIa623sedL62iuraex28cQ6eocK8jiUgb0Jx7iHvo7Y2sLDjAI9NGMSCxq9dxRKSNaOQewt7N381fPtjGjWf1Zeqoo28XJCKBTOUeogr3V/KzeZ8xPCWOX12e6XUcEWljKvcQVFPn444XV+Fzjse/PZaYSM2ziwQbzbmHoN8uWs9nRYf43xvH0rdnZ6/jiEg70Mg9xLy5dhfPLCvglgnpXDL8NK/jiEg7UbmHkIK9n/N/XlnDqNR47rl0iNdxRKQdqdxDRFVtPT/82yrCw43HbhhDVIR2vUgw05x7iHjgjXzyd5Uze0YWKfGdvI4jIu1Mw7cQsGB1MS8u38kPJg3ggiFJXscRkQ6gcg9y+SXl3PvaWsal9eBnkwd5HUdEOojKPYit2nmAaU9+QlxMJI9+ezQR4drdIqFCv+1BatmWvdw4azndu0Qx7/vjSYqL8TqSiHQgHVANQu/m7+aHL64ivWcXnv/eOHqp2EVCjso9yLyeW8xP537G8OQ4nr1lHPGdo7yOJCIeULkHkb8t38GvFqxjXFoP/jrjDLpGa/eKhCr99geJ//1gK79/cwMXDOnF4zeM0c3AREKcyj3AOed46O2NPPb+Vq4YmczD148kUmfFiIQ8lXsA8/kc9/89j2c/2cG0M1J58KrTCQ8zr2OJiB9QuQeounof//7qWl5dVcSt56Rz72VDMVOxi0gDlXsAqq6r5+6Xcnkrr5SfTh7EnRcMVLGLyFeo3ANMZU0dtz+fw9LNe/n1NzO5ZWK615FExA+p3ANIeVUttzy9klU7D/DHa0dwfVaq15FExE+p3APEvsPV3DR7BZt2V/Do9DFcPqK315FExI+p3ANA6aEqbpj1KUUHjvDkTVmcP7iX15FExM+p3P3czn2V3PDXTznweS3P3TKOM/v39DqSiAQAlbsf27y7ghv/upzqOh9/m3kmI1PjvY4kIgFC5e6n1hUf4qbZKwgPM16+bTyDT4v1OpKIBBBdp+6HVhbsZ/qTn9IpMpx5t6vYRaT1NHL3Mx9uKuO257NJ7taJF2aeSbIeZi0iJ6FFI3czu8TMNprZFjO75xhfn2FmZWaW2/hnZttHDX5vrStl5rPZpCd0Ze73x6vYReSkNTtyN7Nw4DFgMlAErDSzhc65/KNWfdk5d0c7ZAwJr60u4ufz1jCiTzeemTGObp0jvY4kIgGsJSP3ccAW59w251wNMAeY2r6xQsvzn+7gJy9/xpnpPXjhe2eq2EXklLWk3FOAwiavixqXHe0aM1tjZq+Y2TGvizez28ws28yyy8rKTiJu8HliyVb+74J1XDS0F7NnnEEXPT1JRNpAW50t83cgzTk3AngHePZYKznnnnTOZTnnshITE9vorQNTbb2PB/+Rzx/e2sCUkck8ceNYPT1JRNpMS4aJxUDTkXifxmVfcs7ta/JyFvDHU48WvIoOVHL3nFxydhzgpvH9+M0Vw/SQDRFpUy0p95VAhpml01Dq04BvN13BzHo753Y1vpwCrG/TlEFkcV4p/zbvM3wO/nv6aKaMTPY6kogEoWbL3TlXZ2Z3AIuBcGC2cy7PzB4Asp1zC4G7zGwKUAfsB2a0Y+aAVF1Xz+8WbeCZZQWcntKNR6ePJi2hi9exRCRImXPOkzfOyspy2dnZnrx3R9u+93PueHEVeSXl3DIhnX+/dDDREZpfF5HWM7Mc51xWc+vp1Ix29npuMffOX0tkRBizbsrioswkryOJSAhQubeTypo67luYx9zsIs5I684j00brilMR6TAq93awsbSCH724iq1lh7nzgoHcfWEGEeG6R5uIdByVextyzjFnZSH3LcwjNiaS5285k4kZCV7HEpEQpHJvI4eO1PLL19byxppdnJORwMPXjyIxNtrrWCISolTup8g5x5vrSvnNwjz2f17Dv31jMD84bwBhuihJRDykcj8FJQeP8OvX1/Hu+j0MS45j9s1ncHqfbl7HEhFRuZ+Mep/juU8KeGjxRnwOfnnZUL47IU0HTUXEb6jcWym/pJxfzF/DZ0WHOG9QIv955XBSe3T2OpaIyFeo3FvoSE09j7y3maeWbiO+UySPTBvFlJHJmGluXUT8j8q9BZZuLuOXr61j5/5KvpWVyi8uG0J85yivY4mIHJfK/QT2Ha7mwX+sZ/7qYvondOGlW89i/ICeXscSEWmWyv04Fqwu5v6/53G4uo67LhjID88fqIdpiEjAULkfQ86O/fz45VzG9I3n99eMYFBSrNeRRERaReV+DH96ZzMJXaN4YeaZdI7SX5GIBB6dmH2UFdv389GWvdx+7gAVu4gELJX7Uf70ziYSukZz41n9vI4iInLSVO5NfLptH59s28cPJg2gU5QOnopI4FK5N3LO8fA7m+gVG80NZ/b1Oo6IyClRuTf6ZOs+Vmzfzw8mDdApjyIS8FTuNIza//TuJpLiopk+TqN2EQl8Knfg4y37WFlwgB/pQiURCRIhX+4Nc+0b6d0thm+dkep1HBGRNhHy5f7h5r2s2nmQH50/kOgIjdpFJDiEdLk75/jTO5tIie/E9VkatYtI8Ajpcl+yqYzcwoZRe1RESP9ViEiQCdlG+2LU3qd7J64d28frOCIibSpky/2fG/awpugQd16gUbuIBJ+QbLUvzmvv26MzV4/RqF1Egk9Ilvs7+btZV1zOHRcMJDI8JP8KRCTIhVyzOef487ub6dezM1ePTvE6johIuwi5cl+ct5v8XeXcdUEGERq1i0iQCql28/kcf353E+kJXZg6KtnrOCIi7Sakyv2tvFI2lFZw94UatYtIcAuZhvP5HI+8u5kBiV24YqRG7SIS3FpU7mZ2iZltNLMtZnbPCda7xsycmWW1XcS2sWjdLjburuCuCzMIDzOv44iItKtmy93MwoHHgEuBTGC6mWUeY71Y4G5geVuHPFX1voYzZDJ6deWbIzRqF5Hg15KR+zhgi3Num3OuBpgDTD3Gev8B/AGoasN8beKNNSVs2XOYuy/SqF1EQkNLyj0FKGzyuqhx2ZfMbAyQ6pz7x4l+kJndZmbZZpZdVlbW6rAnwznH//xzC4OTYrlseO8OeU8REa+d8gFVMwsDHgZ+1ty6zrknnXNZzrmsxMTEU33rFvlgUxmb9xzm9vP6E6ZRu4iEiJaUezHQ9GbnfRqXfSEWGA4sMbMC4Cxgob8cVJ21dDtJcdGaaxeRkNKScl8JZJhZuplFAdOAhV980Tl3yDmX4JxLc86lAZ8CU5xz2e2SuBXW7yrnoy17ufnsNN35UURCSrON55yrA+4AFgPrgbnOuTwze8DMprR3wFPx14+20ykynG+P6+t1FBGRDhXRkpWcc4uARUct+/Vx1p106rFO3Z7yKl7PLebb4/oS3znK6zgiIh0qaOcqnvtkB3U+x3cnpHsdRUSkwwVluR+pqeeF5TuYPDSJtIQuXscREelwQVnur6wq4mBlLbee29/rKCIingi6cvf5HLM/2s7IPt3I6tfd6zgiIp4IunJ/b8Metu/9nJnn9MdMFy2JSGgKunKftXQbKfGduHT4aV5HERHxTFCV+9qiQyzfvp8ZZ6fpYRwiEtKCqgFnfbSNrtERfGtcavMri4gEsaAp95KDR/jHml1864xU4mIivY4jIuKpoCn3Z5cV4HOOGWeneR1FRMRzQVHuh6vreHHFTi49vTepPTp7HUdExHNBUe7zsgupqKpj5kTdakBEBIKg3Ot9jtkfbyerX3dG99VFSyIiEATl/nZeKYX7jzDzHI3aRUS+EPDlPuuj7fTt0ZnJmbpoSUTkCwFd7qt2HiBnxwFumZBGuJ6PKiLypYAu978u3U5cTATXZemiJRGRpgK23Av3V/Lmul18+8x+dIlu0QOlRERCRsCW+9MfFxBmxs1n9/M6ioiI3wnIci+vquXllTv55oje9O7Wyes4IiJ+JyDLfc6KnXxeU8/Mc/SkJRGRYwm4cq+t9/HMxwWc1b8Hw1O6eR1HRMQvBVy5L1q7i5JDVcycqFG7iMjxBFy5d42O4OLMJC4Y0svrKCIifivgziG8cGgSFw5N8jqGiIhfC7iRu4iINE/lLiIShFTuIiJBSOUuIhKEVO4iIkFI5S4iEoRU7iIiQUjlLiIShMw5580bm5UBO07y2xOAvW0YJ9CE8vaH8rZDaG+/tr1BP+dcYnPf4Fm5nwozy3bOZXmdwyuhvP2hvO0Q2tuvbW/dtmtaRkQkCKncRUSCUKCW+5NeB/BYKG9/KG87hPb2a9tbISDn3EVE5MQCdeQuIiInoHIXEQlCAVfuZnaJmW00sy1mdo/XeTqSmRWY2VozyzWzbK/ztDczm21me8xsXZNlPczsHTPb3Pixu5cZ28txtv0+Mytu3P+5ZnaZlxnbi5mlmtn7ZpZvZnlmdnfj8lDZ9038DMsAAAJcSURBVMfb/lbt/4CaczezcGATMBkoAlYC051z+Z4G6yBmVgBkOedC4kIOMzsXOAw855wb3rjsj8B+59zvG//n3t059+9e5mwPx9n2+4DDzrmHvMzW3sysN9DbObfKzGKBHOBKYAahse+Pt/3X04r9H2gj93HAFufcNudcDTAHmOpxJmknzrkPgf1HLZ4KPNv4+bM0/EcfdI6z7SHBObfLObeq8fMKYD2QQujs++Ntf6sEWrmnAIVNXhdxEhsdwBzwtpnlmNltXofxSJJzblfj56VAqD1Q9w4zW9M4bROU0xJNmVkaMBpYTgju+6O2H1qx/wOt3EPdROfcGOBS4EeN/3QPWa5hTjFw5hVP3RPAAGAUsAv4L2/jtC8z6wq8CvzYOVfe9GuhsO+Psf2t2v+BVu7FQGqT130al4UE51xx48c9wGs0TFOFmt2Nc5JfzE3u8ThPh3HO7XbO1TvnfMBTBPH+N7NIGortb865+Y2LQ2bfH2v7W7v/A63cVwIZZpZuZlHANGChx5k6hJl1aTy4gpl1AS4G1p34u4LSQuDmxs9vBl73MEuH+qLYGl1FkO5/MzPgr8B659zDTb4UEvv+eNvf2v0fUGfLADSe/vNnIByY7Zx70ONIHcLM+tMwWgeIAF4M9m03s5eASTTc7nQ38BtgATAX6EvDLaOvd84F3YHH42z7JBr+Se6AAuD2JnPQQcPMJgJLgbWAr3HxvTTMO4fCvj/e9k+nFfs/4MpdRESaF2jTMiIi0gIqdxGRIKRyFxEJQip3EZEgpHIXEQlCKncRkSCkchcRCUL/H0CK1jY73/mQAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZTk-ha_BpMoA"
      },
      "source": [
        "As we can see, the best features seem to be obtained at the middle of the network, after which the performance degrades.\n",
        "\n",
        "Scores:    \n",
        "\n",
        "* ImageGPT-small (features of last layer), max iter 100: 0.52582 on training set, 0.5173 on validation set.\n",
        "* ImageGPT-small (features of last layer), max iter 1000: 0.69892 on training set, 0.6776 on validation set. \n",
        "* ImageGPT-small (features of last layer), max iter 10000: 0.69876 on training set, 0.6776 on validation set.\n",
        "\n",
        "Scores for ImageGPT-small with features of every layer:    \n",
        "```\n",
        "{'features_0': (0.35678, 0.3328),\n",
        " 'features_1': (0.39662, 0.3979),\n",
        " 'features_10': (0.75778, 0.7556),\n",
        " 'features_11': (0.76852, 0.764),\n",
        " 'features_12': (0.78118, 0.7777),\n",
        " 'features_13': (0.79084, 0.784),\n",
        " 'features_14': (0.79096, 0.7823),\n",
        " 'features_15': (0.78704, 0.7828),\n",
        " 'features_16': (0.79426, 0.7862),\n",
        " 'features_17': (0.77646, 0.7679),\n",
        " 'features_18': (0.79218, 0.7823),\n",
        " 'features_19': (0.77898, 0.7686),\n",
        " 'features_2': (0.4538, 0.4555),\n",
        " 'features_20': (0.76906, 0.758),\n",
        " 'features_21': (0.7632, 0.7477),\n",
        " 'features_22': (0.76064, 0.7422),\n",
        " 'features_23': (0.7458, 0.7238),\n",
        " 'features_24': (0.69894, 0.6775),\n",
        " 'features_3': (0.47694, 0.4739),\n",
        " 'features_4': (0.49574, 0.4954),\n",
        " 'features_5': (0.52214, 0.5229),\n",
        " 'features_6': (0.55212, 0.5503),\n",
        " 'features_7': (0.5959, 0.5887),\n",
        " 'features_8': (0.65802, 0.6532),\n",
        " 'features_9': (0.68232, 0.6791)}\n",
        "```\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TXThNYuZoHGL"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}